{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## WideNet prototype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[i] Loaded Parameters:\n",
      " 40000 35 0.2 200 \n",
      " dataset/glove/glove.twitter.27B.200d.txt\n",
      "[i] Importing Modules...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[+] Using Keras version 2.1.4\n",
      "[i] Finished Importing Modules\n",
      "[i] . Reading from csv file...Done!\n",
      "[i] Found 30505 unique tokens.\n",
      "[+] Shape of data tensor: (40000, 30)\n",
      "[+] Shape of label tensor: (40000, 6)\n",
      "[+] Number of entries in each category:\n",
      "[+] Training:\n",
      " [ 7693.  7421.  4874. 10865.  1056.    91.]\n",
      "[+] Validation:\n",
      " [1951. 1849. 1155. 2759.  267.   19.]\n",
      "[i] Loading GloVe from: dataset/glove/glove.twitter.27B.200d.txt ...Done.\n",
      "[+] Proceeding with Embedding Matrix...Completed!\n",
      "Finished running setup.\n"
     ]
    }
   ],
   "source": [
    "%run Setup.ipynb\n",
    "%run ExtraFunctions.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(len(word_index) + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Feed the input vector into the embedding layer, then feed the resulting sequence into a bidirectional LSTM layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "embedded_sequences = embedding_layer(sequence_input)\n",
    "l_lstm1 = Bidirectional(LSTM(8,return_sequences=True))(embedded_sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "inception = [] # append our layers in parallel\n",
    "\n",
    "# kernel 7: best accuracy, some over fitting!\n",
    "l_conv_7 = Conv1D(filters=24,kernel_size=7,activation='relu',kernel_regularizer=regularizers.l2(0.01))(l_lstm1)\n",
    "# kernel 2: good accuracy\n",
    "l_conv_2 = Conv1D(filters=24,kernel_size=2,activation='relu',kernel_regularizer=regularizers.l2(0.01))(l_lstm1)\n",
    "# kernel 6: good accuracy\n",
    "l_conv_6 = Conv1D(filters=24,kernel_size=6,activation='relu',kernel_regularizer=regularizers.l2(0.01))(l_lstm1)\n",
    "# kernel 3: good accuracy, no over fitting\n",
    "l_conv_3 = Conv1D(filters=24,kernel_size=3,activation='relu',kernel_regularizer=regularizers.l2(0.01))(l_lstm1)\n",
    "\n",
    "inception.append(l_conv_7)\n",
    "inception.append(l_conv_2)\n",
    "inception.append(l_conv_6)\n",
    "inception.append(l_conv_3)\n",
    "\n",
    "# poorer performing layers\n",
    "l_conv_4 = Conv1D(filters=24,kernel_size=4,activation='relu',kernel_regularizer=regularizers.l2(0.01))(l_lstm1)\n",
    "l_drop_4 = Dropout(0.6)(l_conv_4)\n",
    "l_conv_5 = Conv1D(filters=24,kernel_size=5,activation='relu',kernel_regularizer=regularizers.l2(0.01))(l_lstm1)\n",
    "l_drop_5 = Dropout(0.6)(l_conv_5)\n",
    "l_conv_8 = Conv1D(filters=24,kernel_size=8,activation='relu',kernel_regularizer=regularizers.l2(0.01))(l_lstm1)\n",
    "l_drop_8 = Dropout(0.6)(l_conv_8)\n",
    "\n",
    "inception.append(l_drop_4)\n",
    "inception.append(l_drop_5)\n",
    "inception.append(l_drop_8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# last parallel layer in inception: max-pooling layer\n",
    "l_pool_i1 = MaxPooling1D(3)(l_lstm1)\n",
    "l_conv_1 = Conv1D(filters=24,kernel_size=1,\n",
    "                    activation='relu',kernel_regularizer=regularizers.l2(0.01))(l_pool_i1)\n",
    "inception.append(l_conv_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "l_merge = Concatenate(axis=1)(inception)\n",
    "l_pool = MaxPooling1D(4)(l_merge)\n",
    "l_drop = Dropout(0.5)(l_pool)\n",
    "l_flat = Flatten()(l_drop)\n",
    "l_dense = Dense(12, activation='relu')(l_flat)\n",
    "preds = Dense(6, activation='softmax')(l_dense)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Model(sequence_input, preds)\n",
    "adadelta = optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=None, decay=0.0)\n",
    "lr_metric = get_lr_metric(adadelta)\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=adadelta,\n",
    "              metrics=['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -r logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tensorboard = callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=16, write_grads=True , write_graph=True)\n",
    "model_checkpoints = callbacks.ModelCheckpoint(\"checkpoint.h5\", monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=False, mode='auto', period=1)\n",
    "lr_schedule = callbacks.LearningRateScheduler(initial_boost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 30)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 30, 200)      6101200     input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_1 (Bidirectional) (None, 30, 16)       13376       embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_5 (Conv1D)               (None, 27, 24)       1560        bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_6 (Conv1D)               (None, 26, 24)       1944        bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_7 (Conv1D)               (None, 23, 24)       3096        bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1D)  (None, 10, 16)       0           bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, 24, 24)       2712        bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, 29, 24)       792         bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_3 (Conv1D)               (None, 25, 24)       2328        bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_4 (Conv1D)               (None, 28, 24)       1176        bidirectional_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 27, 24)       0           conv1d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 26, 24)       0           conv1d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 23, 24)       0           conv1d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_8 (Conv1D)               (None, 10, 24)       408         max_pooling1d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 192, 24)      0           conv1d_1[0][0]                   \n",
      "                                                                 conv1d_2[0][0]                   \n",
      "                                                                 conv1d_3[0][0]                   \n",
      "                                                                 conv1d_4[0][0]                   \n",
      "                                                                 dropout_1[0][0]                  \n",
      "                                                                 dropout_2[0][0]                  \n",
      "                                                                 dropout_3[0][0]                  \n",
      "                                                                 conv1d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1D)  (None, 48, 24)       0           concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 48, 24)       0           max_pooling1d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 1152)         0           dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 12)           13836       flatten_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 6)            78          dense_1[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 6,142,506\n",
      "Trainable params: 41,306\n",
      "Non-trainable params: 6,101,200\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()\n",
    "model.save('WideNet.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Progress:\n",
      "Train on 32000 samples, validate on 8000 samples\n",
      "Epoch 1/200\n",
      "32000/32000 [==============================] - 21s 664us/step - loss: 1.5478 - acc: 0.3359 - val_loss: 1.4605 - val_acc: 0.3449\n",
      "Epoch 2/200\n",
      "32000/32000 [==============================] - 21s 662us/step - loss: 1.4669 - acc: 0.3395 - val_loss: 1.4577 - val_acc: 0.3449\n",
      "Epoch 3/200\n",
      "32000/32000 [==============================] - 21s 658us/step - loss: 1.4670 - acc: 0.3395 - val_loss: 1.4578 - val_acc: 0.3449\n",
      "Epoch 4/200\n",
      "32000/32000 [==============================] - 21s 670us/step - loss: 1.4669 - acc: 0.3395 - val_loss: 1.4585 - val_acc: 0.3449\n",
      "Epoch 5/200\n",
      "32000/32000 [==============================] - 21s 661us/step - loss: 1.4661 - acc: 0.3395 - val_loss: 1.4590 - val_acc: 0.3449\n",
      "Epoch 6/200\n",
      "32000/32000 [==============================] - 21s 656us/step - loss: 1.4664 - acc: 0.3395 - val_loss: 1.4570 - val_acc: 0.3449\n",
      "Epoch 7/200\n",
      "32000/32000 [==============================] - 21s 654us/step - loss: 1.4649 - acc: 0.3395 - val_loss: 1.4546 - val_acc: 0.3449\n",
      "Epoch 8/200\n",
      "32000/32000 [==============================] - 21s 656us/step - loss: 1.4589 - acc: 0.3398 - val_loss: 1.4439 - val_acc: 0.3476\n",
      "Epoch 9/200\n",
      "32000/32000 [==============================] - 21s 659us/step - loss: 1.3938 - acc: 0.3976 - val_loss: 1.3465 - val_acc: 0.4231\n",
      "Epoch 10/200\n",
      "32000/32000 [==============================] - 21s 657us/step - loss: 1.3416 - acc: 0.4332 - val_loss: 1.3293 - val_acc: 0.4419\n",
      "Epoch 11/200\n",
      "32000/32000 [==============================] - 21s 657us/step - loss: 1.3254 - acc: 0.4408 - val_loss: 1.3097 - val_acc: 0.4491\n",
      "Epoch 12/200\n",
      "32000/32000 [==============================] - 21s 657us/step - loss: 1.3129 - acc: 0.4577 - val_loss: 1.3041 - val_acc: 0.4789\n",
      "Epoch 13/200\n",
      "32000/32000 [==============================] - 21s 659us/step - loss: 1.3012 - acc: 0.4654 - val_loss: 1.3148 - val_acc: 0.4539\n",
      "Epoch 14/200\n",
      "32000/32000 [==============================] - 21s 648us/step - loss: 1.2908 - acc: 0.4737 - val_loss: 1.2909 - val_acc: 0.4675\n",
      "Epoch 15/200\n",
      "32000/32000 [==============================] - 21s 642us/step - loss: 1.2819 - acc: 0.4800 - val_loss: 1.2839 - val_acc: 0.4805\n",
      "Epoch 16/200\n",
      "32000/32000 [==============================] - 21s 646us/step - loss: 1.2723 - acc: 0.4833 - val_loss: 1.2795 - val_acc: 0.4796\n",
      "Epoch 17/200\n",
      "32000/32000 [==============================] - 21s 652us/step - loss: 1.2632 - acc: 0.4912 - val_loss: 1.2741 - val_acc: 0.4849\n",
      "Epoch 18/200\n",
      "32000/32000 [==============================] - 21s 656us/step - loss: 1.2575 - acc: 0.4887 - val_loss: 1.2719 - val_acc: 0.4834\n",
      "Epoch 19/200\n",
      "32000/32000 [==============================] - 21s 658us/step - loss: 1.2518 - acc: 0.4954 - val_loss: 1.2741 - val_acc: 0.4874\n",
      "Epoch 20/200\n",
      "32000/32000 [==============================] - 21s 650us/step - loss: 1.2444 - acc: 0.4981 - val_loss: 1.2734 - val_acc: 0.4893\n",
      "Epoch 21/200\n",
      "32000/32000 [==============================] - 21s 655us/step - loss: 1.2434 - acc: 0.4987 - val_loss: 1.2708 - val_acc: 0.4841\n",
      "Epoch 22/200\n",
      "32000/32000 [==============================] - 21s 659us/step - loss: 1.2375 - acc: 0.5019 - val_loss: 1.2923 - val_acc: 0.4764\n",
      "Epoch 23/200\n",
      "32000/32000 [==============================] - 21s 664us/step - loss: 1.2338 - acc: 0.5037 - val_loss: 1.2730 - val_acc: 0.4886\n",
      "Epoch 24/200\n",
      "32000/32000 [==============================] - 21s 650us/step - loss: 1.2281 - acc: 0.5056 - val_loss: 1.2825 - val_acc: 0.4816\n",
      "Epoch 25/200\n",
      "32000/32000 [==============================] - 21s 654us/step - loss: 1.2247 - acc: 0.5080 - val_loss: 1.2708 - val_acc: 0.4889\n",
      "Epoch 26/200\n",
      "32000/32000 [==============================] - 21s 652us/step - loss: 1.2193 - acc: 0.5107 - val_loss: 1.2663 - val_acc: 0.4882\n",
      "Epoch 27/200\n",
      "32000/32000 [==============================] - 21s 654us/step - loss: 1.2166 - acc: 0.5104 - val_loss: 1.2821 - val_acc: 0.4877\n",
      "Epoch 28/200\n",
      "32000/32000 [==============================] - 21s 652us/step - loss: 1.2115 - acc: 0.5148 - val_loss: 1.2805 - val_acc: 0.4881\n",
      "Epoch 29/200\n",
      "32000/32000 [==============================] - 21s 652us/step - loss: 1.2063 - acc: 0.5177 - val_loss: 1.3189 - val_acc: 0.4655\n",
      "Epoch 30/200\n",
      "32000/32000 [==============================] - 21s 655us/step - loss: 1.2041 - acc: 0.5162 - val_loss: 1.2804 - val_acc: 0.4849\n",
      "Epoch 31/200\n",
      "32000/32000 [==============================] - 21s 660us/step - loss: 1.2008 - acc: 0.5218 - val_loss: 1.2762 - val_acc: 0.4895\n",
      "Epoch 32/200\n",
      "32000/32000 [==============================] - 21s 655us/step - loss: 1.1956 - acc: 0.5241 - val_loss: 1.3115 - val_acc: 0.4766\n",
      "Epoch 33/200\n",
      "32000/32000 [==============================] - 21s 654us/step - loss: 1.1932 - acc: 0.5280 - val_loss: 1.2999 - val_acc: 0.4740\n",
      "Epoch 34/200\n",
      "32000/32000 [==============================] - 21s 652us/step - loss: 1.1895 - acc: 0.5304 - val_loss: 1.2878 - val_acc: 0.4830\n",
      "Epoch 35/200\n",
      "32000/32000 [==============================] - 21s 653us/step - loss: 1.1890 - acc: 0.5296 - val_loss: 1.2928 - val_acc: 0.4814\n",
      "Epoch 36/200\n",
      "32000/32000 [==============================] - 21s 670us/step - loss: 1.1829 - acc: 0.5355 - val_loss: 1.2828 - val_acc: 0.4886\n",
      "Epoch 37/200\n",
      "32000/32000 [==============================] - 21s 664us/step - loss: 1.1781 - acc: 0.5356 - val_loss: 1.2966 - val_acc: 0.4834\n",
      "Epoch 38/200\n",
      "32000/32000 [==============================] - 21s 661us/step - loss: 1.1766 - acc: 0.5408 - val_loss: 1.2913 - val_acc: 0.4901\n",
      "Epoch 39/200\n",
      "32000/32000 [==============================] - 21s 664us/step - loss: 1.1735 - acc: 0.5419 - val_loss: 1.2962 - val_acc: 0.4839\n",
      "Epoch 40/200\n",
      "32000/32000 [==============================] - 21s 659us/step - loss: 1.1695 - acc: 0.5459 - val_loss: 1.3020 - val_acc: 0.4818\n",
      "Epoch 41/200\n",
      "32000/32000 [==============================] - 21s 658us/step - loss: 1.1657 - acc: 0.5465 - val_loss: 1.2992 - val_acc: 0.4822\n",
      "Epoch 42/200\n",
      "32000/32000 [==============================] - 21s 658us/step - loss: 1.1641 - acc: 0.5496 - val_loss: 1.3186 - val_acc: 0.4824\n",
      "Epoch 43/200\n",
      "32000/32000 [==============================] - 21s 654us/step - loss: 1.1622 - acc: 0.5498 - val_loss: 1.2928 - val_acc: 0.4929\n",
      "Epoch 44/200\n",
      "32000/32000 [==============================] - 21s 656us/step - loss: 1.1576 - acc: 0.5542 - val_loss: 1.3155 - val_acc: 0.4899\n",
      "Epoch 45/200\n",
      "32000/32000 [==============================] - 21s 662us/step - loss: 1.1534 - acc: 0.5567 - val_loss: 1.3084 - val_acc: 0.4922\n",
      "Epoch 46/200\n",
      "32000/32000 [==============================] - 21s 661us/step - loss: 1.1518 - acc: 0.5574 - val_loss: 1.3118 - val_acc: 0.4919\n",
      "Epoch 47/200\n",
      "32000/32000 [==============================] - 21s 658us/step - loss: 1.1458 - acc: 0.5610 - val_loss: 1.3052 - val_acc: 0.4945\n",
      "Epoch 48/200\n",
      "32000/32000 [==============================] - 21s 653us/step - loss: 1.1468 - acc: 0.5616 - val_loss: 1.3047 - val_acc: 0.4777\n",
      "Epoch 49/200\n",
      "32000/32000 [==============================] - 21s 643us/step - loss: 1.1431 - acc: 0.5650 - val_loss: 1.3289 - val_acc: 0.4871\n",
      "Epoch 50/200\n",
      "32000/32000 [==============================] - 21s 641us/step - loss: 1.1380 - acc: 0.5661 - val_loss: 1.3274 - val_acc: 0.4914\n",
      "Epoch 51/200\n",
      "32000/32000 [==============================] - 20s 638us/step - loss: 1.1357 - acc: 0.5671 - val_loss: 1.3250 - val_acc: 0.4884\n",
      "Epoch 52/200\n",
      "32000/32000 [==============================] - 21s 644us/step - loss: 1.1333 - acc: 0.5670 - val_loss: 1.3267 - val_acc: 0.4929\n",
      "Epoch 53/200\n",
      "32000/32000 [==============================] - 21s 649us/step - loss: 1.1285 - acc: 0.5712 - val_loss: 1.3370 - val_acc: 0.4851\n",
      "Epoch 54/200\n",
      "32000/32000 [==============================] - 21s 649us/step - loss: 1.1258 - acc: 0.5722 - val_loss: 1.3182 - val_acc: 0.4851\n",
      "Epoch 55/200\n",
      "32000/32000 [==============================] - 21s 656us/step - loss: 1.1256 - acc: 0.5720 - val_loss: 1.3096 - val_acc: 0.5004\n",
      "Epoch 56/200\n",
      "32000/32000 [==============================] - 21s 653us/step - loss: 1.1239 - acc: 0.5749 - val_loss: 1.3452 - val_acc: 0.4804\n",
      "Epoch 57/200\n",
      "32000/32000 [==============================] - 21s 655us/step - loss: 1.1183 - acc: 0.5768 - val_loss: 1.3226 - val_acc: 0.4922\n",
      "Epoch 58/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32000/32000 [==============================] - 21s 647us/step - loss: 1.1172 - acc: 0.5778 - val_loss: 1.3210 - val_acc: 0.4906\n",
      "Epoch 59/200\n",
      "32000/32000 [==============================] - 21s 649us/step - loss: 1.1173 - acc: 0.5786 - val_loss: 1.3353 - val_acc: 0.4834\n",
      "Epoch 60/200\n",
      "32000/32000 [==============================] - 21s 648us/step - loss: 1.1130 - acc: 0.5787 - val_loss: 1.3287 - val_acc: 0.4785\n",
      "Epoch 61/200\n",
      "32000/32000 [==============================] - 21s 654us/step - loss: 1.1121 - acc: 0.5819 - val_loss: 1.3473 - val_acc: 0.4854\n",
      "Epoch 62/200\n",
      "32000/32000 [==============================] - 21s 649us/step - loss: 1.1108 - acc: 0.5805 - val_loss: 1.3605 - val_acc: 0.4810\n",
      "Epoch 63/200\n",
      "32000/32000 [==============================] - 21s 650us/step - loss: 1.1074 - acc: 0.5833 - val_loss: 1.3630 - val_acc: 0.4809\n",
      "Epoch 64/200\n",
      "32000/32000 [==============================] - 21s 649us/step - loss: 1.1064 - acc: 0.5831 - val_loss: 1.3450 - val_acc: 0.4839\n",
      "Epoch 65/200\n",
      "32000/32000 [==============================] - 21s 652us/step - loss: 1.1034 - acc: 0.5845 - val_loss: 1.3846 - val_acc: 0.4726\n",
      "Epoch 66/200\n",
      "32000/32000 [==============================] - 21s 654us/step - loss: 1.1001 - acc: 0.5871 - val_loss: 1.3570 - val_acc: 0.4777\n",
      "Epoch 67/200\n",
      "32000/32000 [==============================] - 21s 653us/step - loss: 1.0995 - acc: 0.5864 - val_loss: 1.3345 - val_acc: 0.4921\n",
      "Epoch 68/200\n",
      "32000/32000 [==============================] - 21s 655us/step - loss: 1.0958 - acc: 0.5880 - val_loss: 1.3596 - val_acc: 0.4812\n",
      "Epoch 69/200\n",
      "32000/32000 [==============================] - 21s 653us/step - loss: 1.0966 - acc: 0.5872 - val_loss: 1.3554 - val_acc: 0.4859\n",
      "Epoch 70/200\n",
      "32000/32000 [==============================] - 21s 655us/step - loss: 1.0931 - acc: 0.5903 - val_loss: 1.3481 - val_acc: 0.4876\n",
      "Epoch 71/200\n",
      "32000/32000 [==============================] - 21s 655us/step - loss: 1.0920 - acc: 0.5887 - val_loss: 1.3768 - val_acc: 0.4780\n",
      "Epoch 72/200\n",
      "32000/32000 [==============================] - 21s 654us/step - loss: 1.0893 - acc: 0.5908 - val_loss: 1.3582 - val_acc: 0.4771\n",
      "Epoch 73/200\n",
      "32000/32000 [==============================] - 21s 651us/step - loss: 1.0871 - acc: 0.5933 - val_loss: 1.3410 - val_acc: 0.4862\n",
      "Epoch 74/200\n",
      "32000/32000 [==============================] - 21s 656us/step - loss: 1.0845 - acc: 0.5937 - val_loss: 1.3620 - val_acc: 0.4856\n",
      "Epoch 75/200\n",
      "32000/32000 [==============================] - 21s 656us/step - loss: 1.0832 - acc: 0.5945 - val_loss: 1.3678 - val_acc: 0.4887\n",
      "Epoch 76/200\n",
      "32000/32000 [==============================] - 21s 661us/step - loss: 1.0832 - acc: 0.5930 - val_loss: 1.3629 - val_acc: 0.4896\n",
      "Epoch 77/200\n",
      "32000/32000 [==============================] - 21s 658us/step - loss: 1.0796 - acc: 0.5957 - val_loss: 1.3595 - val_acc: 0.4862\n",
      "Epoch 78/200\n",
      "32000/32000 [==============================] - 21s 654us/step - loss: 1.0754 - acc: 0.5977 - val_loss: 1.3936 - val_acc: 0.4841\n",
      "Epoch 79/200\n",
      "32000/32000 [==============================] - 21s 655us/step - loss: 1.0750 - acc: 0.5991 - val_loss: 1.3533 - val_acc: 0.4870\n",
      "Epoch 80/200\n",
      "32000/32000 [==============================] - 21s 654us/step - loss: 1.0775 - acc: 0.5969 - val_loss: 1.3956 - val_acc: 0.4827\n",
      "Epoch 81/200\n",
      "32000/32000 [==============================] - 21s 655us/step - loss: 1.0715 - acc: 0.5994 - val_loss: 1.4037 - val_acc: 0.4865\n",
      "Epoch 82/200\n",
      "32000/32000 [==============================] - 21s 652us/step - loss: 1.0704 - acc: 0.6004 - val_loss: 1.3944 - val_acc: 0.4766\n",
      "Epoch 83/200\n",
      "32000/32000 [==============================] - 21s 653us/step - loss: 1.0704 - acc: 0.5997 - val_loss: 1.4007 - val_acc: 0.4756\n",
      "Epoch 84/200\n",
      "32000/32000 [==============================] - 20s 639us/step - loss: 1.0682 - acc: 0.5984 - val_loss: 1.3792 - val_acc: 0.4834\n",
      "Epoch 85/200\n",
      "32000/32000 [==============================] - 20s 635us/step - loss: 1.0642 - acc: 0.6026 - val_loss: 1.3673 - val_acc: 0.4811\n",
      "Epoch 86/200\n",
      "32000/32000 [==============================] - 20s 641us/step - loss: 1.0636 - acc: 0.6040 - val_loss: 1.4098 - val_acc: 0.4761\n",
      "Epoch 87/200\n",
      "32000/32000 [==============================] - 21s 642us/step - loss: 1.0626 - acc: 0.6052 - val_loss: 1.3706 - val_acc: 0.4802\n",
      "Epoch 88/200\n",
      "32000/32000 [==============================] - 21s 655us/step - loss: 1.0602 - acc: 0.6062 - val_loss: 1.4227 - val_acc: 0.4709\n",
      "Epoch 89/200\n",
      "32000/32000 [==============================] - 21s 646us/step - loss: 1.0555 - acc: 0.6087 - val_loss: 1.3992 - val_acc: 0.4794\n",
      "Epoch 90/200\n",
      "32000/32000 [==============================] - 21s 660us/step - loss: 1.0565 - acc: 0.6057 - val_loss: 1.3910 - val_acc: 0.4799\n",
      "Epoch 91/200\n",
      "32000/32000 [==============================] - 21s 653us/step - loss: 1.0554 - acc: 0.6068 - val_loss: 1.4354 - val_acc: 0.4682\n",
      "Epoch 92/200\n",
      "32000/32000 [==============================] - 21s 648us/step - loss: 1.0531 - acc: 0.6087 - val_loss: 1.4001 - val_acc: 0.4750\n",
      "Epoch 93/200\n",
      "32000/32000 [==============================] - 21s 651us/step - loss: 1.0513 - acc: 0.6094 - val_loss: 1.3794 - val_acc: 0.4829\n",
      "Epoch 94/200\n",
      "32000/32000 [==============================] - 21s 650us/step - loss: 1.0510 - acc: 0.6074 - val_loss: 1.3901 - val_acc: 0.4830\n",
      "Epoch 95/200\n",
      "32000/32000 [==============================] - 21s 652us/step - loss: 1.0506 - acc: 0.6114 - val_loss: 1.3821 - val_acc: 0.4840\n",
      "Epoch 96/200\n",
      "32000/32000 [==============================] - 21s 655us/step - loss: 1.0481 - acc: 0.6102 - val_loss: 1.3970 - val_acc: 0.4824\n",
      "Epoch 97/200\n",
      "32000/32000 [==============================] - 21s 647us/step - loss: 1.0447 - acc: 0.6127 - val_loss: 1.4020 - val_acc: 0.4815\n",
      "Epoch 98/200\n",
      "32000/32000 [==============================] - 20s 614us/step - loss: 1.0446 - acc: 0.6153 - val_loss: 1.4218 - val_acc: 0.4757\n",
      "Epoch 99/200\n",
      "32000/32000 [==============================] - 20s 618us/step - loss: 1.0427 - acc: 0.6129 - val_loss: 1.4389 - val_acc: 0.4799\n",
      "Epoch 100/200\n",
      "32000/32000 [==============================] - 20s 616us/step - loss: 1.0417 - acc: 0.6151 - val_loss: 1.4179 - val_acc: 0.4782\n",
      "Epoch 101/200\n",
      "32000/32000 [==============================] - 20s 618us/step - loss: 1.0402 - acc: 0.6138 - val_loss: 1.4349 - val_acc: 0.4738\n",
      "Epoch 102/200\n",
      "32000/32000 [==============================] - 20s 616us/step - loss: 1.0373 - acc: 0.6149 - val_loss: 1.4490 - val_acc: 0.4794\n",
      "Epoch 103/200\n",
      "32000/32000 [==============================] - 20s 616us/step - loss: 1.0373 - acc: 0.6161 - val_loss: 1.4250 - val_acc: 0.4736\n",
      "Epoch 104/200\n",
      "32000/32000 [==============================] - 20s 613us/step - loss: 1.0376 - acc: 0.6164 - val_loss: 1.3921 - val_acc: 0.4781\n",
      "Epoch 105/200\n",
      "32000/32000 [==============================] - 20s 618us/step - loss: 1.0339 - acc: 0.6194 - val_loss: 1.4458 - val_acc: 0.4731\n",
      "Epoch 106/200\n",
      "32000/32000 [==============================] - 20s 614us/step - loss: 1.0323 - acc: 0.6194 - val_loss: 1.4469 - val_acc: 0.4732\n",
      "Epoch 107/200\n",
      "32000/32000 [==============================] - 20s 614us/step - loss: 1.0333 - acc: 0.6187 - val_loss: 1.4446 - val_acc: 0.4734\n",
      "Epoch 108/200\n",
      "32000/32000 [==============================] - 19s 609us/step - loss: 1.0338 - acc: 0.6164 - val_loss: 1.4432 - val_acc: 0.4764\n",
      "Epoch 109/200\n",
      "32000/32000 [==============================] - 20s 613us/step - loss: 1.0301 - acc: 0.6183 - val_loss: 1.4400 - val_acc: 0.4789\n",
      "Epoch 110/200\n",
      "32000/32000 [==============================] - 20s 613us/step - loss: 1.0291 - acc: 0.6214 - val_loss: 1.4174 - val_acc: 0.4764\n",
      "Epoch 111/200\n",
      "32000/32000 [==============================] - 20s 615us/step - loss: 1.0302 - acc: 0.6206 - val_loss: 1.4115 - val_acc: 0.4809\n",
      "Epoch 112/200\n",
      "32000/32000 [==============================] - 20s 611us/step - loss: 1.0263 - acc: 0.6213 - val_loss: 1.4278 - val_acc: 0.4744\n",
      "Epoch 113/200\n",
      "32000/32000 [==============================] - 20s 612us/step - loss: 1.0267 - acc: 0.6199 - val_loss: 1.4523 - val_acc: 0.4767\n",
      "Epoch 114/200\n",
      "32000/32000 [==============================] - 20s 613us/step - loss: 1.0229 - acc: 0.6237 - val_loss: 1.4139 - val_acc: 0.4744\n",
      "Epoch 115/200\n",
      "32000/32000 [==============================] - 20s 610us/step - loss: 1.0219 - acc: 0.6238 - val_loss: 1.4051 - val_acc: 0.4815\n",
      "Epoch 116/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32000/32000 [==============================] - 19s 609us/step - loss: 1.0232 - acc: 0.6219 - val_loss: 1.4328 - val_acc: 0.4614\n",
      "Epoch 117/200\n",
      "32000/32000 [==============================] - 20s 613us/step - loss: 1.0209 - acc: 0.6228 - val_loss: 1.4560 - val_acc: 0.4746\n",
      "Epoch 118/200\n",
      "32000/32000 [==============================] - 20s 614us/step - loss: 1.0193 - acc: 0.6233 - val_loss: 1.4218 - val_acc: 0.4755\n",
      "Epoch 119/200\n",
      "32000/32000 [==============================] - 20s 613us/step - loss: 1.0170 - acc: 0.6217 - val_loss: 1.4511 - val_acc: 0.4701\n",
      "Epoch 120/200\n",
      "32000/32000 [==============================] - 20s 615us/step - loss: 1.0198 - acc: 0.6235 - val_loss: 1.4591 - val_acc: 0.4706\n",
      "Epoch 121/200\n",
      "32000/32000 [==============================] - 19s 609us/step - loss: 1.0189 - acc: 0.6248 - val_loss: 1.4312 - val_acc: 0.4695\n",
      "Epoch 122/200\n",
      "32000/32000 [==============================] - 20s 611us/step - loss: 1.0162 - acc: 0.6256 - val_loss: 1.4487 - val_acc: 0.4694\n",
      "Epoch 123/200\n",
      "32000/32000 [==============================] - 19s 609us/step - loss: 1.0169 - acc: 0.6243 - val_loss: 1.4246 - val_acc: 0.4769\n",
      "Epoch 124/200\n",
      "32000/32000 [==============================] - 20s 610us/step - loss: 1.0157 - acc: 0.6259 - val_loss: 1.4305 - val_acc: 0.4685\n",
      "Epoch 125/200\n",
      "32000/32000 [==============================] - 20s 613us/step - loss: 1.0120 - acc: 0.6287 - val_loss: 1.4574 - val_acc: 0.4726\n",
      "Epoch 126/200\n",
      "32000/32000 [==============================] - 20s 614us/step - loss: 1.0133 - acc: 0.6272 - val_loss: 1.4921 - val_acc: 0.4750\n",
      "Epoch 127/200\n",
      "32000/32000 [==============================] - 20s 612us/step - loss: 1.0130 - acc: 0.6264 - val_loss: 1.4615 - val_acc: 0.4737\n",
      "Epoch 128/200\n",
      "32000/32000 [==============================] - 20s 610us/step - loss: 1.0121 - acc: 0.6293 - val_loss: 1.4655 - val_acc: 0.4701\n",
      "Epoch 129/200\n",
      "32000/32000 [==============================] - 20s 610us/step - loss: 1.0120 - acc: 0.6250 - val_loss: 1.4602 - val_acc: 0.4749\n",
      "Epoch 130/200\n",
      "32000/32000 [==============================] - 20s 611us/step - loss: 1.0089 - acc: 0.6277 - val_loss: 1.4607 - val_acc: 0.4739\n",
      "Epoch 131/200\n",
      "32000/32000 [==============================] - 20s 615us/step - loss: 1.0095 - acc: 0.6305 - val_loss: 1.4745 - val_acc: 0.4682\n",
      "Epoch 132/200\n",
      "32000/32000 [==============================] - 20s 612us/step - loss: 1.0078 - acc: 0.6291 - val_loss: 1.4555 - val_acc: 0.4700\n",
      "Epoch 133/200\n",
      "32000/32000 [==============================] - 20s 610us/step - loss: 1.0039 - acc: 0.6302 - val_loss: 1.4497 - val_acc: 0.4779\n",
      "Epoch 134/200\n",
      "32000/32000 [==============================] - 20s 613us/step - loss: 1.0085 - acc: 0.6280 - val_loss: 1.4701 - val_acc: 0.4745\n",
      "Epoch 135/200\n",
      "32000/32000 [==============================] - 20s 610us/step - loss: 1.0050 - acc: 0.6299 - val_loss: 1.4642 - val_acc: 0.4762\n",
      "Epoch 136/200\n",
      "32000/32000 [==============================] - 20s 616us/step - loss: 1.0058 - acc: 0.6306 - val_loss: 1.4541 - val_acc: 0.4709\n",
      "Epoch 137/200\n",
      "32000/32000 [==============================] - 20s 610us/step - loss: 1.0051 - acc: 0.6318 - val_loss: 1.4850 - val_acc: 0.4722\n",
      "Epoch 138/200\n",
      "32000/32000 [==============================] - 19s 606us/step - loss: 1.0013 - acc: 0.6314 - val_loss: 1.4608 - val_acc: 0.4728\n",
      "Epoch 139/200\n",
      "32000/32000 [==============================] - 20s 616us/step - loss: 1.0016 - acc: 0.6322 - val_loss: 1.4873 - val_acc: 0.4716\n",
      "Epoch 140/200\n",
      "32000/32000 [==============================] - 19s 607us/step - loss: 1.0016 - acc: 0.6313 - val_loss: 1.4993 - val_acc: 0.4697\n",
      "Epoch 141/200\n",
      "32000/32000 [==============================] - 19s 609us/step - loss: 0.9983 - acc: 0.6369 - val_loss: 1.4635 - val_acc: 0.4729\n",
      "Epoch 142/200\n",
      "32000/32000 [==============================] - 20s 614us/step - loss: 0.9983 - acc: 0.6323 - val_loss: 1.4681 - val_acc: 0.4766\n",
      "Epoch 143/200\n",
      "32000/32000 [==============================] - 20s 611us/step - loss: 0.9976 - acc: 0.6341 - val_loss: 1.4801 - val_acc: 0.4724\n",
      "Epoch 144/200\n",
      "32000/32000 [==============================] - 20s 612us/step - loss: 0.9971 - acc: 0.6336 - val_loss: 1.5001 - val_acc: 0.4680\n",
      "Epoch 145/200\n",
      "32000/32000 [==============================] - 20s 613us/step - loss: 0.9961 - acc: 0.6363 - val_loss: 1.4885 - val_acc: 0.4711\n",
      "Epoch 146/200\n",
      "32000/32000 [==============================] - 19s 608us/step - loss: 0.9966 - acc: 0.6348 - val_loss: 1.4800 - val_acc: 0.4695\n",
      "Epoch 147/200\n",
      "32000/32000 [==============================] - 19s 609us/step - loss: 0.9945 - acc: 0.6353 - val_loss: 1.5225 - val_acc: 0.4659\n",
      "Epoch 148/200\n",
      "32000/32000 [==============================] - 19s 608us/step - loss: 0.9962 - acc: 0.6345 - val_loss: 1.4804 - val_acc: 0.4702\n",
      "Epoch 149/200\n",
      "32000/32000 [==============================] - 20s 610us/step - loss: 0.9951 - acc: 0.6363 - val_loss: 1.4779 - val_acc: 0.4692\n",
      "Epoch 150/200\n",
      "32000/32000 [==============================] - 19s 609us/step - loss: 0.9904 - acc: 0.6361 - val_loss: 1.5085 - val_acc: 0.4760\n",
      "Epoch 151/200\n",
      "32000/32000 [==============================] - 20s 615us/step - loss: 0.9918 - acc: 0.6366 - val_loss: 1.4634 - val_acc: 0.4662\n",
      "Epoch 152/200\n",
      "32000/32000 [==============================] - 20s 613us/step - loss: 0.9902 - acc: 0.6381 - val_loss: 1.4973 - val_acc: 0.4697\n",
      "Epoch 153/200\n",
      "32000/32000 [==============================] - 20s 612us/step - loss: 0.9909 - acc: 0.6378 - val_loss: 1.5543 - val_acc: 0.4552\n",
      "Epoch 154/200\n",
      "32000/32000 [==============================] - 20s 613us/step - loss: 0.9925 - acc: 0.6352 - val_loss: 1.4939 - val_acc: 0.4714\n",
      "Epoch 155/200\n",
      "32000/32000 [==============================] - 19s 604us/step - loss: 0.9911 - acc: 0.6352 - val_loss: 1.4921 - val_acc: 0.4691\n",
      "Epoch 156/200\n",
      "32000/32000 [==============================] - 19s 608us/step - loss: 0.9884 - acc: 0.6349 - val_loss: 1.4839 - val_acc: 0.4651\n",
      "Epoch 157/200\n",
      "32000/32000 [==============================] - 20s 612us/step - loss: 0.9899 - acc: 0.6378 - val_loss: 1.4587 - val_acc: 0.4717\n",
      "Epoch 158/200\n",
      "32000/32000 [==============================] - 20s 611us/step - loss: 0.9886 - acc: 0.6386 - val_loss: 1.5214 - val_acc: 0.4705\n",
      "Epoch 159/200\n",
      "32000/32000 [==============================] - 20s 611us/step - loss: 0.9858 - acc: 0.6382 - val_loss: 1.4835 - val_acc: 0.4653\n",
      "Epoch 160/200\n",
      "32000/32000 [==============================] - 20s 610us/step - loss: 0.9848 - acc: 0.6417 - val_loss: 1.5034 - val_acc: 0.4681\n",
      "Epoch 161/200\n",
      "32000/32000 [==============================] - 20s 611us/step - loss: 0.9844 - acc: 0.6412 - val_loss: 1.5043 - val_acc: 0.4664\n",
      "Epoch 162/200\n",
      "32000/32000 [==============================] - 20s 611us/step - loss: 0.9839 - acc: 0.6418 - val_loss: 1.4859 - val_acc: 0.4689\n",
      "Epoch 163/200\n",
      "32000/32000 [==============================] - 20s 611us/step - loss: 0.9842 - acc: 0.6410 - val_loss: 1.4756 - val_acc: 0.4716\n",
      "Epoch 164/200\n",
      "32000/32000 [==============================] - 19s 608us/step - loss: 0.9844 - acc: 0.6390 - val_loss: 1.4837 - val_acc: 0.4699\n",
      "Epoch 165/200\n",
      "32000/32000 [==============================] - 20s 612us/step - loss: 0.9836 - acc: 0.6389 - val_loss: 1.4810 - val_acc: 0.4716\n",
      "Epoch 166/200\n",
      "32000/32000 [==============================] - 20s 612us/step - loss: 0.9851 - acc: 0.6389 - val_loss: 1.4768 - val_acc: 0.4635\n",
      "Epoch 167/200\n",
      "32000/32000 [==============================] - 20s 611us/step - loss: 0.9832 - acc: 0.6394 - val_loss: 1.5141 - val_acc: 0.4677\n",
      "Epoch 168/200\n",
      "32000/32000 [==============================] - 20s 610us/step - loss: 0.9845 - acc: 0.6399 - val_loss: 1.5120 - val_acc: 0.4677\n",
      "Epoch 169/200\n",
      "32000/32000 [==============================] - 20s 610us/step - loss: 0.9828 - acc: 0.6399 - val_loss: 1.4922 - val_acc: 0.4671\n",
      "Epoch 170/200\n",
      "32000/32000 [==============================] - 19s 609us/step - loss: 0.9828 - acc: 0.6409 - val_loss: 1.4938 - val_acc: 0.4701\n",
      "Epoch 171/200\n",
      "32000/32000 [==============================] - 20s 610us/step - loss: 0.9776 - acc: 0.6429 - val_loss: 1.5025 - val_acc: 0.4669\n",
      "Epoch 172/200\n",
      "32000/32000 [==============================] - 19s 608us/step - loss: 0.9805 - acc: 0.6407 - val_loss: 1.5519 - val_acc: 0.4686\n",
      "Epoch 173/200\n",
      "32000/32000 [==============================] - 20s 616us/step - loss: 0.9816 - acc: 0.6403 - val_loss: 1.4922 - val_acc: 0.4701\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 174/200\n",
      "32000/32000 [==============================] - 20s 610us/step - loss: 0.9794 - acc: 0.6423 - val_loss: 1.5132 - val_acc: 0.4665\n",
      "Epoch 175/200\n",
      "32000/32000 [==============================] - 20s 612us/step - loss: 0.9765 - acc: 0.6448 - val_loss: 1.5548 - val_acc: 0.4681\n",
      "Epoch 176/200\n",
      "32000/32000 [==============================] - 20s 611us/step - loss: 0.9743 - acc: 0.6442 - val_loss: 1.5273 - val_acc: 0.4690\n",
      "Epoch 177/200\n",
      "32000/32000 [==============================] - 20s 610us/step - loss: 0.9781 - acc: 0.6420 - val_loss: 1.5480 - val_acc: 0.4569\n",
      "Epoch 178/200\n",
      "32000/32000 [==============================] - 20s 611us/step - loss: 0.9751 - acc: 0.6440 - val_loss: 1.5051 - val_acc: 0.4716\n",
      "Epoch 179/200\n",
      "32000/32000 [==============================] - 20s 613us/step - loss: 0.9742 - acc: 0.6454 - val_loss: 1.5166 - val_acc: 0.4673\n",
      "Epoch 180/200\n",
      "32000/32000 [==============================] - 19s 609us/step - loss: 0.9766 - acc: 0.6420 - val_loss: 1.4936 - val_acc: 0.4646\n",
      "Epoch 181/200\n",
      "32000/32000 [==============================] - 20s 613us/step - loss: 0.9752 - acc: 0.6431 - val_loss: 1.5229 - val_acc: 0.4680\n",
      "Epoch 182/200\n",
      "32000/32000 [==============================] - 20s 616us/step - loss: 0.9713 - acc: 0.6443 - val_loss: 1.5613 - val_acc: 0.4598\n",
      "Epoch 183/200\n",
      "32000/32000 [==============================] - 20s 611us/step - loss: 0.9753 - acc: 0.6450 - val_loss: 1.5190 - val_acc: 0.4642\n",
      "Epoch 184/200\n",
      "32000/32000 [==============================] - 20s 613us/step - loss: 0.9738 - acc: 0.6438 - val_loss: 1.5095 - val_acc: 0.4691\n",
      "Epoch 185/200\n",
      "32000/32000 [==============================] - 20s 612us/step - loss: 0.9711 - acc: 0.6466 - val_loss: 1.5384 - val_acc: 0.4661\n",
      "Epoch 186/200\n",
      "32000/32000 [==============================] - 19s 609us/step - loss: 0.9725 - acc: 0.6445 - val_loss: 1.5403 - val_acc: 0.4625\n",
      "Epoch 187/200\n",
      "32000/32000 [==============================] - 19s 609us/step - loss: 0.9719 - acc: 0.6453 - val_loss: 1.5706 - val_acc: 0.4619\n",
      "Epoch 188/200\n",
      "32000/32000 [==============================] - 20s 616us/step - loss: 0.9704 - acc: 0.6466 - val_loss: 1.5385 - val_acc: 0.4686\n",
      "Epoch 189/200\n",
      "32000/32000 [==============================] - 20s 613us/step - loss: 0.9671 - acc: 0.6473 - val_loss: 1.5431 - val_acc: 0.4642\n",
      "Epoch 190/200\n",
      "32000/32000 [==============================] - 19s 604us/step - loss: 0.9717 - acc: 0.6453 - val_loss: 1.5610 - val_acc: 0.4580\n",
      "Epoch 191/200\n",
      "32000/32000 [==============================] - 19s 609us/step - loss: 0.9684 - acc: 0.6458 - val_loss: 1.5651 - val_acc: 0.4655\n",
      "Epoch 192/200\n",
      "32000/32000 [==============================] - 20s 611us/step - loss: 0.9666 - acc: 0.6470 - val_loss: 1.5520 - val_acc: 0.4662\n",
      "Epoch 193/200\n",
      "32000/32000 [==============================] - 19s 609us/step - loss: 0.9675 - acc: 0.6463 - val_loss: 1.5140 - val_acc: 0.4651\n",
      "Epoch 194/200\n",
      "32000/32000 [==============================] - 20s 610us/step - loss: 0.9671 - acc: 0.6482 - val_loss: 1.5260 - val_acc: 0.4640\n",
      "Epoch 195/200\n",
      "32000/32000 [==============================] - 19s 605us/step - loss: 0.9655 - acc: 0.6470 - val_loss: 1.5506 - val_acc: 0.4733\n",
      "Epoch 196/200\n",
      "32000/32000 [==============================] - 19s 606us/step - loss: 0.9674 - acc: 0.6469 - val_loss: 1.5149 - val_acc: 0.4689\n",
      "Epoch 197/200\n",
      "32000/32000 [==============================] - 19s 608us/step - loss: 0.9654 - acc: 0.6469 - val_loss: 1.5197 - val_acc: 0.4647\n",
      "Epoch 198/200\n",
      "32000/32000 [==============================] - 20s 611us/step - loss: 0.9652 - acc: 0.6489 - val_loss: 1.5666 - val_acc: 0.4647\n",
      "Epoch 199/200\n",
      "32000/32000 [==============================] - 19s 608us/step - loss: 0.9642 - acc: 0.6480 - val_loss: 1.5304 - val_acc: 0.4644\n",
      "Epoch 200/200\n",
      "32000/32000 [==============================] - 19s 606us/step - loss: 0.9657 - acc: 0.6493 - val_loss: 1.5505 - val_acc: 0.4656\n"
     ]
    }
   ],
   "source": [
    "print(\"Training Progress:\")\n",
    "model_log = model.fit(x_train, y_train, validation_data=(x_val, y_val),\n",
    "          epochs=200, batch_size=50,\n",
    "          callbacks=[tensorboard, lr_schedule])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "model.save('WideNet.h5')\n",
    "pd.DataFrame(model_log.history).to_csv(\"history-inception.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
